package cn.net.withub.dataCollector.web.collector;

import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.net.withub.dataCollector.common.model.TCollectorConfig;
import cn.net.withub.dataCollector.common.model.TCollectorData;
import cn.net.withub.dataCollector.common.utils.CreateNewKey;
import cn.net.withub.dataCollector.common.utils.HTMLSpirit;
import cn.net.withub.dataCollector.web.service.CollectorService;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.regex.Pattern;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.context.WebApplicationContext;

/**
 * 新浪网国内新闻
 */
public class CollectorUtilsSina extends CrawlerBase {

    public CollectorUtilsSina(String crawlPath, boolean autoParse, WebApplicationContext wac, TCollectorConfig config) {
        super(crawlPath, autoParse, wac, config);
        webAppliction = wac;
        if(webAppliction !=null) collectorService = (CollectorService) webAppliction.getBean("collectorService");
        if(config != null) tCollectorConfig = config;
        sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm");
    }

    public void visit(Page page, CrawlDatums crawlDatums) {
        String url = page.url();
        String html = page.html();
        String contentType = page.contentType();
        String nextUrl = "";
        String title = "", fbsj = "", djcs = "", xxly = "";//信息来源
        log.info("--------------------------URL:" + url);
        try{
            if (page.matchUrl("http://news.sina.com.cn/china.*")) {//国内新闻
                Elements elements = page.select(".content .left .blk12 a");
                for (Element e : elements) {
                    nextUrl = e.attr("abs:href");
                    if (!Pattern.matches("http://news.sina.com.cn/.*.shtml", nextUrl)) {
                        continue;
                    }
                    title = e.text();
                    //crawlDatums.add(nextUrl, "newsDetail");
                    TCollectorData tCollectorData = (TCollectorData) collectorService.load(TCollectorData.class, "title ='" + title + "'");
                    if(tCollectorData==null){
                        crawlDatums.add(new CrawlDatum(nextUrl, "newsDetail").meta("title", title).meta("preUrL", url));
                    }
                }
            } else if (page.matchType("newsDetail")) {//单条新闻详情页面
                Elements element = page.select("#top_bar_wrap .date-source");
                Elements contentEle = page.select("#article");//内容包含广告   需过滤
                fbsj = element.get(0).child(0).text();
                //fbsj = fbsj.replace("年","-").replace("月","-").replace("日","");
                //replace是字符串或者字符的替换  replaceAll是基于正则表达式的替换
                //fbsj = fbsj.substring(0,10).replaceAll("[年月]","-").replaceAll("[日]","");
                fbsj = fbsj.replaceAll("[年月]","-").replaceAll("[日]","");
                xxly = element.get(0).child(1).text();
                //去掉编辑
                contentEle.select("p[class='show_author']").remove();
                //类似于专题  回顾  推荐,去掉
                contentEle.select(".wap_special").remove();
                //Elements pa =contentEle.select("p a");
                Elements pstrong =contentEle.select("p strong");
                for (int j = 0, lenj = pstrong.size(); j < lenj; j++) {
                    Element e = pstrong.get(j).parent().nextElementSibling();
                    if (e == null) continue;
                    String temp = e.html();
                    if(temp.contains("</a>") || temp.contains("</strong>")){
                        pstrong.get(j).parent().remove();
                    }
                }
                while (contentEle.select("p a").size()>0){
                    contentEle.select("p a").get(0).parent().remove();
                }
                /*for (int i = 0, len = pa.size(); i < len; i++) {
                    //有这种两个a的情况,会把父节的父节删除报错   <p>　　<a href="http://news.sina.com.cn/s/2018-07-23/doc-ihftenhy7616921.shtml" target="_blank">央视：</a><a href="http://news.sina.com.cn/s/2018-07-23/doc-ihftenhy7616921.shtml" target="_blank">疫苗事件带给我们的三个沉重的思考</a></p>
                    //if (pa.get(i).parent().select("p")) {
                    boolean aa = pa.get(i).parent().tag().toString().equals("p");
                    if(pa.get(i).parent().tag().toString().equals("p")){
                        pa.get(i).parent().remove();
                    }
                }*/
                String content = contentEle.outerHtml();
                String text = HTMLSpirit.getTextFromTHML(content);
                if("".equals(text.trim())) return;//无文字  基本全是视频
                Elements imgs = contentEle.select("img[src]");
                String id = new CreateNewKey().createId();
                //遍历所有html中所有图片
                for (Element img : imgs) {
                    String src = img.attr("abs:src");
                    //此处图片地址包含中文会报错
                    src = fmtUrl(src);
                    log.info("img url:\n"+src);
                    img.attr("style", "display: block;margin : 0 auto");
                    img.parent().attr("style", "text-align: center;");
                    String imageName = src.substring(src.lastIndexOf("/")+1,src.length());
                    String serverFilePath = "";
                    try {
                        //serverFilePath = HttpClientUtils.getImage(src);
                        serverFilePath = FtpUtil.getImage(src, "");
                        //img.attr("src", serverFilePath);
                        saveElement(id, imageName, src, serverFilePath);
                    } catch (IOException e) {
                        log.error("出错src:" + src + ";父级路径:" + url);
                        e.printStackTrace();
                    }
                }
                title = getStr(page.meta("title"));
                String preUrL = getStr(page.meta("preUrL"));
                int zt=1;
                try {
                    //保存TCollectorData
                    saveData(id, title, fbsj, xxly, "html", url, "", "", "", "");
                    //保存TCollectorContent
                    saveContent(id,content,text);
                    //保存最大发布时间
                    saveMaxFbsj(fbsj);
                    zt=1;
                }catch (Exception e){
                    zt=0;
                    e.printStackTrace();
                }finally {
                    saveLog(title,fbsj,preUrL,url,zt);
                }
            }

        }catch (Exception e){
            e.printStackTrace();
        }
    }

}
